# -*- coding: utf-8 -*-
import logging
import datetime
import time
from time import sleep, strftime
import csv
from html.parser import HTMLParser
from bs4 import BeautifulSoup

from ibapi.client import EClient
from ibapi.wrapper import EWrapper
from ibapi.contract import Contract
from ibapi.wrapper import BarData
news = {}

class IBNewsApp(EClient, EWrapper):
    def __init__(self):
        EWrapper.__init__(self)
        EClient.__init__(self, wrapper=self)
        self.requestid = 2000

    def nextValidId(self, orderId:int):
        print("Next Valid Id:", orderId)
        self.start()

    def historicalNews(self, requestId:int, time:str, providerCode:str, articleId:str, headline:str):
        print("Historical News:", requestId, time, providerCode, articleId, headline)
        self.requestid += 1
        news[self.requestid] = {
            "time": time,
            "providerCode": providerCode,
            "articleId": articleId,
            "headline": headline
        }
        self.reqNewsArticle(self.requestid, providerCode= providerCode, articleId=articleId, newsArticleOptions=[])

    def historicalNewsEnd(self, requestId:int, hasMore:bool):
        print("Historical News End:", requestId, hasMore)

    def newsArticle(self, requestId:int, articleType:int, articleText:str):
        print("News Article:", requestId,articleType, articleText)
        it = news[requestId]
        with (open(f"./data/news_aapl.csv", "a", newline="") as csvfile):
            writer = csv.writer(csvfile)
            text = ''
            if articleText.startswith("<html>"):
                parser = BeautifulSoup(articleText, "html.parser")
                text = parser.getText()
            else:
                arr = articleText.split("&#10;")
                issueIdx = -1
                copyrightIdx = -1
                for i in range(len(arr)):
                    item = arr[i]
                    if type(item) != str:
                        continue
                    b = item.startswith("Issuance", 0)
                    if b:
                        issueIdx = i
                if issueIdx >= 0 :
                    arr.pop(issueIdx)
                for i in range(len(arr)):
                    item = arr[i]
                    if type(item) != str:
                        continue
                    b = item.startswith("Copyright", 0)
                    if b:
                        copyrightIdx = i
                if copyrightIdx >= 0:
                    arr.pop(copyrightIdx)
                text = ''.join(arr)
            t = time.strptime(it['time'], "%Y-%m-%d %H:%M:%S.0")
            t1 = strftime("%Y-%m-%d", t)
            t2 = strftime("%H:%M:%S", t)
            writer.writerow([articleType, it['providerCode'], it['time'], t1, t2, it['articleId'], it['headline'], text ])
    def start(self):
        self.getNews()
    def getNews(self):
        reqId = 1
        t = datetime.datetime.now()
        diff = datetime.timedelta(days=-730)
        t = t + diff
        datestr = t.strftime("%Y%m%d %H:%M:%S")
        self.reqHistoricalNews(reqId, 265598, "BRFG+BRFUPDN+DJNL", datestr, "", 200,
                              historicalNewsOptions=[])



if __name__ == "__main__":
    app = IBNewsApp()
    app.connect("127.0.0.1", 7496, 0)
    app.run()
